package com.csdnindex;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.util.HashMap;
import java.util.Map;

import org.apache.log4j.LogManager;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.csdn.HttpUtils;
import com.csdn.UserAgentMiddle;

public class GetcsdnIndex {
	private static org.apache.log4j.Logger log = LogManager.getLogger(GetcsdnIndex.class);
	public  static String url = "http://blog.csdn.net"; // /?&page=3
	private static final int settime = 3000;
	public static void main(String[] args) throws IOException {
		Connection getAllExperts = Jsoup
				.connect(url)
				.userAgent(
						"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0")
				.timeout(settime).method(Connection.Method.GET);

		Document ex = getAllExperts.get();
		Element exbody = ex.body();

		String total = exbody.getElementsByClass("page_nav")
				.select("span:eq(0)").text();
		String regex = ".+共(\\d+)页";
		total = total.replaceAll(regex, "$1");
		int totalPage = Integer.parseInt(total);
		getExperts(totalPage); 
		
		
	}
	public static void getExperts(int totalPage) throws IOException {
		int count =0;
		for (int i = 1; i <= totalPage; i++) {
			try {
				Connection getAllExperts = Jsoup
						.connect(url+"/?&page="+i)
						.userAgent(UserAgentMiddle.Agent())
						.timeout(settime).method(Connection.Method.GET);

				Document ex = getAllExperts.get();
				Element exbody = ex.body();
				Element  el= exbody.getElementById("listBot");
				if(el  == null){
					Elements elss = exbody.getElementsByClass("blog_list_wrap");
					if (!elss.isEmpty()) {
						el = elss.get(0);
					}
				}
				if(count ==4){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.94");
					System.getProperties().setProperty("http.proxyPort", "8080");		
					
				}
				if(count ==8){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "61.130.97.212");
					System.getProperties().setProperty("http.proxyPort", "8099");		
					
				}
				if(count ==14){
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "111.1.3.34");
					System.getProperties().setProperty("http.proxyPort", "8000");		
					count = 0;
				}
				
				
				Elements  els = el.getElementsByClass("tracking-ad");
				Document bodys = null;
				for (Element element : els) {
					Element linkNode = (element.select("a")).get(0);
					String href = linkNode.attr("href");
					log.info("正在爬取："+i +"---url:"+href);
					String nickname = href.substring(href.indexOf("csdn.net")+9,href.length());
					nickname = nickname.substring(0, nickname.indexOf("/"));
					 bodys = getdetails(href);
					 if(bodys == null){
						 continue;
					 }
					 connectionUrl(linkNode.text(), getTopic(bodys),
							 nickname, 2,
								5, null, "fxbbs",getfourmName(bodys));
					
				}
	 
				
			}catch(ConnectException e1){
				e1.printStackTrace();
				System.setProperty("http.maxRedirects", "50");
				System.getProperties().setProperty("proxySet", "true");
				System.getProperties().setProperty("http.proxyHost", "183.222.102.108");
				System.getProperties().setProperty("http.proxyPort", "8080");	
				
				log.error("超时："+i);
			}
			catch (Exception e) {
				 e.printStackTrace();
				 log.error("错误："+e.getMessage() +"总数："+i);
				 continue;
			}
			
			
			
		}
	}
	// 获取帖子内容
	public static String getTopic(Document bodys) throws IOException {
		Elements es = bodys.getElementsByClass("article_content");
		if(!es.hasText()){
			  es = bodys.getElementsByClass("skin_detail");
		}
		return es.toString();
	}
	
	//获取帖子类型
	private static String getfourmName(Document bodys) {
		Elements es = bodys.getElementsByClass("category_r");
		if (es.hasText()) {
			for (Element element : es) {
				Element linkNode = null;
				Elements ls = null;
				try {
					ls = element.select("label span");
					if(ls.isEmpty()){
						ls = element.select("label em");
					}
					linkNode = ls.get(0);
				} catch (Exception e) {
					e.printStackTrace();
					return null;
				}
				String fournName = linkNode.text();
				if (fournName.contains("（")) {
					fournName = fournName.substring(0, fournName.indexOf("（"));
				} else if (fournName.contains("(")) {
					fournName = fournName.substring(0, fournName.indexOf("("));
				}
				return fournName;
			}

		} else {
			es = bodys.getElementsByClass("detail_p");
			for (Element element : es) {

				Element linkNode = null;
				try {
					linkNode = (element.select("label em")).get(0);
				} catch (Exception e) {
					return null;
				}

				String fournName = linkNode.text();
				if (fournName.contains("（")) {
					fournName = fournName.substring(0, fournName.indexOf("（"));
				} else if (fournName.contains("(")) {
					fournName = fournName.substring(0, fournName.indexOf("("));
				}
				return fournName;
			}
		}

		return null;
	}
	// 获取帖子主体
	public static Document getdetails(String url) throws IOException {
		Document body = null;
		try {
			return body = Jsoup.connect(url)
					.userAgent(UserAgentMiddle.Agent()).timeout(settime)
					.method(Connection.Method.GET).get();
		} catch (Exception e) {
		 
		}
		return body;
	}
	
	// 发送到bbs
	public static void connectionUrl(String title, String content,
			String userName, Integer forumId, Integer postTypeId,
			Integer category, String ps,String fourmName) throws UnsupportedEncodingException {

		Map<String, String> params = new HashMap<String, String>();
		params.put("title", title);
		params.put("content", content);
		params.put("userName", userName);

		params.put("forumId", forumId + "");
		params.put("postTypeId", postTypeId + "");
		params.put("ps", ps);
		params.put("froumName", fourmName == null ?"1":fourmName);
		String result = HttpUtils.sendPostMessage(params, "UTF-8");
		log.debug("-result->>" + result);
 
	}
	
}
